{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# 处理重复数据\n",
    "### duplicated筛选、drop_duplicates丢弃"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  data1  data2\n",
      "0     a      2\n",
      "1     a      1\n",
      "2     a      2\n",
      "3     a      2\n",
      "4     b      0\n",
      "5     b      2\n",
      "6     b      2\n",
      "7     b      1\n",
      "--------------------------------------------------\n",
      "0    False\n",
      "1    False\n",
      "2     True\n",
      "3     True\n",
      "4    False\n",
      "5    False\n",
      "6     True\n",
      "7    False\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "np.random.seed(123)\n",
    "df_obj = pd.DataFrame({'data1' : ['a'] * 4 + ['b'] * 4,\n",
    "                       'data2' : np.random.randint(0, 4, 8)})\n",
    "print(df_obj)\n",
    "print(\"-\"*50)\n",
    "\n",
    "# 检查所有特征，如果两个样本的特征值相同，则重复\n",
    "# 第一次出现的不算重复，之后出现的都算\n",
    "print(df_obj.duplicated())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-05-06T02:03:54.714531300Z",
     "start_time": "2024-05-06T02:03:54.655961600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "  data1  data2\n0     a      2\n1     a      1\n4     b      0\n5     b      2\n7     b      1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>data2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>a</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>a</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>b</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 取出不重复的样本，不改变原数据\n",
    "df_obj[~df_obj.duplicated()]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-05-06T02:03:54.734749600Z",
     "start_time": "2024-05-06T02:03:54.676906600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    False\n",
      "1    False\n",
      "2     True\n",
      "3     True\n",
      "4    False\n",
      "5     True\n",
      "6     True\n",
      "7     True\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "# 检查某一特征，即这个特征下每个值只能出现一次\n",
    "print(df_obj.duplicated('data2'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-05-06T02:03:54.735746400Z",
     "start_time": "2024-05-06T02:03:54.689906600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  data1  data2\n",
      "0     a      2\n",
      "1     a      1\n",
      "4     b      0\n",
      "5     b      2\n",
      "7     b      1\n"
     ]
    },
    {
     "data": {
      "text/plain": "  data1  data2\n0     a      2\n1     a      1\n4     b      0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>data2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>a</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>a</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 丢弃重复行，如果要改变原数据，则需要inplace=True\n",
    "print(df_obj.drop_duplicates())\n",
    "df_obj.drop_duplicates('data2',inplace=True)\n",
    "df_obj"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-06T02:03:57.162546900Z",
     "start_time": "2024-05-06T02:03:57.130185300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "   data1 data2\n0    NaN     1\n1    NaN     2\n2    NaN     3\n3    NaN     5",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>data2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>NaN</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_obj1 = pd.DataFrame({'data1' :[np.nan] * 4,\n",
    "                       'data2' :list('1235')})\n",
    "df_obj1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-05-06T02:03:31.613419100Z",
     "start_time": "2024-05-06T02:03:31.519157300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    False\n",
      "1     True\n",
      "2     True\n",
      "3     True\n",
      "dtype: bool\n",
      "--------------------------------------------------\n",
      "   data1 data2\n",
      "0    NaN     1\n",
      "--------------------------------------------------\n",
      "   data1 data2\n",
      "0    NaN     1\n",
      "1    NaN     2\n",
      "2    NaN     3\n",
      "3    NaN     5\n"
     ]
    }
   ],
   "source": [
    "# 在pd的duplicated认为空值和空值相等的\n",
    "print(df_obj1.duplicated('data1'))\n",
    "print(\"-\"*50)\n",
    "\n",
    "print(df_obj1.drop_duplicates('data1'))\n",
    "print(\"-\"*50)\n",
    "print(df_obj1)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-05-03T13:50:00.833660100Z",
     "start_time": "2024-05-03T13:50:00.796530500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   data1 data2\n",
      "0    NaN     1\n"
     ]
    }
   ],
   "source": [
    "# 要真正丢弃，则需要inplace=True\n",
    "df_obj1.drop_duplicates('data1',inplace=True)\n",
    "print(df_obj1)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-03T13:50:09.845511300Z",
     "start_time": "2024-05-03T13:50:09.827450200Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# map与applymap一样，map只能用于series，applymap只能用于df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    2\n",
      "1    2\n",
      "2    6\n",
      "3    1\n",
      "4    3\n",
      "5    9\n",
      "6    6\n",
      "7    1\n",
      "8    0\n",
      "9    1\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "np.random.seed(123)\n",
    "ser_obj = pd.Series(np.random.randint(0,10,10))  #series 用map\n",
    "print(ser_obj)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-05-03T13:50:14.008069Z",
     "start_time": "2024-05-03T13:50:13.977149500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     4\n",
      "1     4\n",
      "2    36\n",
      "3     1\n",
      "4     9\n",
      "5    81\n",
      "6    36\n",
      "7     1\n",
      "8     0\n",
      "9     1\n",
      "dtype: int64\n",
      "--------------------------------------------------\n",
      "0     4\n",
      "1     4\n",
      "2    36\n",
      "3     1\n",
      "4     9\n",
      "5    81\n",
      "6    36\n",
      "7     1\n",
      "8     0\n",
      "9     1\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# apply也可以用于Series\n",
    "# 它俩都不会改变原数据\n",
    "print(ser_obj.map(lambda x : x ** 2))\n",
    "print(\"-\"*50)\n",
    "\n",
    "print(ser_obj.apply(lambda x : x ** 2))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-05-03T13:51:17.497817300Z",
     "start_time": "2024-05-03T13:51:17.479852300Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [
     "# 11 数据合并(pd.concat)\n"
    ],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
